{
 "cells": [
  {
   "cell_type": "code",
   "id": "initial_id",
   "metadata": {
    "collapsed": true,
    "ExecuteTime": {
     "end_time": "2025-05-07T15:27:15.499308Z",
     "start_time": "2025-05-07T15:07:25.451929Z"
    }
   },
   "source": [
    "#数据集下载\n",
    "from modelscope.msdatasets import MsDataset\n",
    "ds =  MsDataset.load('AI-ModelScope/deepctrl-sft-custom_data', subset_name='default', split='train')\n",
    "#您可按需配置 subset_name、split，参照“快速使用”示例代码"
   ],
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2025-05-07 23:07:25,484 - modelscope - WARNING - Use trust_remote_code=True. Will invoke codes from deepctrl-sft-data. Please make sure that you can trust the external codes.\n",
      "2025-05-07 23:07:26,166 - modelscope - WARNING - Use trust_remote_code=True. Will invoke codes from AI-ModelScope/deepctrl-sft-data. Please make sure that you can trust the external codes.\n",
      "2025-05-07 23:07:26,168 - modelscope - WARNING - Use trust_remote_code=True. Will invoke codes from AI-ModelScope/deepctrl-sft-data. Please make sure that you can trust the external codes.\n",
      "2025-05-07 23:07:26,170 - modelscope - WARNING - Use trust_remote_code=True. Will invoke codes from AI-ModelScope/deepctrl-sft-data. Please make sure that you can trust the external codes.\n",
      "Downloading [README.md]: 100%|██████████| 5.68k/5.68k [00:00<?, ?B/s]\n",
      "2025-05-07 23:07:28,215 - modelscope - INFO - storing https://www.modelscope.cn/api/v1/datasets/AI-ModelScope/deepctrl-sft-data/repo?Source=SDK&Revision=master&FilePath=README.md&View=False in cache at C:\\Users\\51165\\.cache\\modelscope\\hub\\datasets\\3e72063cb026390c4bd1cc0b7e2e98661b675b94aad6f43cf0d90b15a535290d\n",
      "2025-05-07 23:07:28,215 - modelscope - INFO - creating metadata file for C:\\Users\\51165\\.cache\\modelscope\\hub\\datasets\\3e72063cb026390c4bd1cc0b7e2e98661b675b94aad6f43cf0d90b15a535290d\n",
      "Downloading data: 17.2GB [19:00, 15.1MB/s]\n",
      "2025-05-07 23:26:36,392 - modelscope - INFO - storing https://www.modelscope.cn/api/v1/datasets/AI-ModelScope/deepctrl-sft-data/repo?Source=SDK&Revision=master&FilePath=sft_data_zh.jsonl in cache at C:\\Users\\51165\\.cache\\modelscope\\hub\\datasets/downloads\\c66ef118119ece0d7b49f25c296d2ed1b13ec6334705ce88f5f768377cf6abab\n",
      "2025-05-07 23:26:36,393 - modelscope - INFO - creating metadata file for C:\\Users\\51165\\.cache\\modelscope\\hub\\datasets/downloads\\c66ef118119ece0d7b49f25c296d2ed1b13ec6334705ce88f5f768377cf6abab\n",
      "Generating train split: 11381621 examples [00:36, 313067.03 examples/s]\n"
     ]
    }
   ],
   "execution_count": 8
  },
  {
   "cell_type": "code",
   "id": "d4eef72439f624d3",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-05-07T15:27:15.526818Z",
     "start_time": "2025-05-07T15:27:15.520565Z"
    }
   },
   "source": [
    "ds"
   ],
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Dataset({\n",
       "    features: ['id', 'instruction', 'input', 'output', 'history', 'language', 'data_source', 'input_len', 'output_len', 'num_utter', 'type', 'type_keyword'],\n",
       "    num_rows: 11381621\n",
       "})"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "execution_count": 9
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "f3603074da7ba197",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2025-05-07 21:32:29,954 - modelscope - WARNING - Use trust_remote_code=True. Will invoke codes from deepctrl-sft-data. Please make sure that you can trust the external codes.\n",
      "2025-05-07 21:32:30,604 - modelscope - WARNING - Use trust_remote_code=True. Will invoke codes from deepctrl/deepctrl-sft-data. Please make sure that you can trust the external codes.\n",
      "2025-05-07 21:32:30,604 - modelscope - WARNING - Use trust_remote_code=True. Will invoke codes from deepctrl/deepctrl-sft-data. Please make sure that you can trust the external codes.\n",
      "2025-05-07 21:32:30,604 - modelscope - WARNING - Use trust_remote_code=True. Will invoke codes from deepctrl/deepctrl-sft-data. Please make sure that you can trust the external codes.\n"
     ]
    }
   ],
   "source": [
    "from modelscope.msdatasets import MsDataset\n",
    "dataset_name = 'deepctrl/deepctrl-sft-custom_data'\n",
    "ds = MsDataset.load(dataset_name)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "f099a871",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "DatasetDict({\n",
       "    train: Dataset({\n",
       "        features: ['id', 'instruction', 'input', 'output', 'history', 'language', 'data_source', 'input_len', 'output_len', 'num_utter', 'type', 'type_keyword'],\n",
       "        num_rows: 10\n",
       "    })\n",
       "})"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ds"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4d150736",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "a311",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.11"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
